"""
实时流处理管道
协调整个实时视频流处理流程，管理500路摄像头的并发处理
"""
import asyncio
import logging
from typing import Dict, List, Optional, Any, Set
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor
from collections import defaultdict, deque
import time
import uuid

from sqlalchemy.orm import Session
from core.database import SessionLocal
from core.config import get_settings
from services.video_processing.video_stream_service import VideoStreamService
from services.ai_infrastructure.ai_batch_processor import AIBatchProcessor
from services.infrastructure.sync_scheduler import get_sync_scheduler
from services.ai_infrastructure.gpu_resource_manager import gpu_resource_manager
from services.ai_infrastructure.ai_service_router import AIServiceRouter
from repositories.camera_repository import CameraRepository
from schemas.video_stream import VideoStreamRequest
from schemas.gpu_resource import GPUResourceRequest, TaskPriority

logger = logging.getLogger(__name__)
settings = get_settings()


class CameraStreamInfo:
    """摄像头流信息"""
    
    def __init__(self, camera_id: str, priority: int = 5):
        self.camera_id = camera_id
        self.priority = priority
        self.status = "inactive"  # inactive, starting, active, error, stopping
        self.worker_id: Optional[str] = None
        self.started_at: Optional[datetime] = None
        self.last_frame_time: Optional[datetime] = None
        self.frames_processed = 0
        self.error_count = 0
        self.last_error: Optional[str] = None
        self.ai_algorithms: List[str] = []
        self.processing_fps = settings.VIDEO_PROCESSING_FPS
        self.health_status = "unknown"  # healthy, unhealthy, unknown


class StreamWorker:
    """流处理工作器"""
    
    def __init__(self, worker_id: str, max_cameras: int = 100):
        self.worker_id = worker_id
        self.max_cameras = max_cameras
        self.assigned_cameras: Set[str] = set()
        self.load_factor = 0.0
        self.status = "idle"  # idle, running, overloaded, error
        self.created_at = datetime.now()
        self.last_activity = datetime.now()
        self.processed_frames = 0
        self.error_count = 0


class RealTimeStreamPipeline:
    """实时流处理管道"""
    
    def __init__(self):
        self.is_running = False
        self.camera_streams: Dict[str, CameraStreamInfo] = {}
        self.stream_workers: Dict[str, StreamWorker] = {}
        self.worker_tasks: Dict[str, asyncio.Task] = {}
        
        # 服务依赖
        self.video_service = VideoStreamService()
        self.ai_processor = None  # 将在初始化时设置
        self.ai_router = AIServiceRouter()
        self.sync_scheduler = get_sync_scheduler()
        
        # 线程池配置
        max_concurrent_streams = getattr(settings, 'MAX_CONCURRENT_STREAMS', 500)
        self.max_workers = max(1, max_concurrent_streams // 100)  # 每个worker处理100路摄像头，最少1个worker
        self.thread_pool = ThreadPoolExecutor(max_workers=self.max_workers)
        
        # 性能监控
        self.performance_metrics = {
            "total_cameras": 0,
            "active_cameras": 0,
            "total_frames_processed": 0,
            "average_fps": 0.0,
            "system_load": 0.0,
            "gpu_utilization": 0.0,
            "error_rate": 0.0,
            "uptime": 0.0
        }
        
        # 管理任务
        self._monitor_task: Optional[asyncio.Task] = None
        self._load_balancer_task: Optional[asyncio.Task] = None
        self._health_check_task: Optional[asyncio.Task] = None
        
        # 启动时间
        self.started_at: Optional[datetime] = None
        
    async def initialize(self):
        """初始化实时流处理管道"""
        try:
            logger.info("初始化实时流处理管道")
            
            # 初始化AI批处理器
            ai_config = {
                "batch_size": settings.AI_BATCH_SIZE,
                "max_wait_time": settings.AI_MAX_WAIT_TIME,
                "max_workers": settings.AI_MAX_WORKERS,
                "enable_gpu": True,
                "enable_multiprocessing": True
            }
            self.ai_processor = AIBatchProcessor(ai_config)
            
            # 设置AI服务路由器引用
            self.ai_processor.ai_service_router = self.ai_router
            self.ai_processor.gpu_manager = gpu_resource_manager
            
            # 启动AI批处理器
            self.ai_processor.start_processing()
            
            # 创建初始工作器
            await self._create_initial_workers()
            
            logger.info(f"实时流处理管道初始化完成，创建了 {len(self.stream_workers)} 个工作器")
            
        except Exception as e:
            logger.error(f"初始化实时流处理管道失败: {e}")
            raise
    
    async def start(self):
        """启动实时流处理管道"""
        if self.is_running:
            logger.warning("实时流处理管道已在运行")
            return
        
        try:
            logger.info("启动实时流处理管道")
            
            self.is_running = True
            self.started_at = datetime.now()
            
            # 启动管理任务
            self._monitor_task = asyncio.create_task(self._monitoring_loop())
            self._load_balancer_task = asyncio.create_task(self._load_balancing_loop())
            self._health_check_task = asyncio.create_task(self._health_check_loop())
            
            # 启动工作器任务
            for worker_id in self.stream_workers.keys():
                await self._start_worker(worker_id)
            
            logger.info("实时流处理管道启动成功")
            
        except Exception as e:
            logger.error(f"启动实时流处理管道失败: {e}")
            self.is_running = False
            raise
    
    async def stop(self):
        """停止实时流处理管道"""
        if not self.is_running:
            return
        
        try:
            logger.info("停止实时流处理管道")
            
            self.is_running = False
            
            # 停止所有摄像头流
            camera_ids = list(self.camera_streams.keys())
            await self.batch_stop_cameras(camera_ids)
            
            # 停止管理任务
            for task in [self._monitor_task, self._load_balancer_task, self._health_check_task]:
                if task and not task.done():
                    task.cancel()
                    try:
                        await task
                    except asyncio.CancelledError:
                        pass
            
            # 停止工作器任务
            for worker_id in list(self.worker_tasks.keys()):
                await self._stop_worker(worker_id)
            
            # 停止AI批处理器
            if self.ai_processor:
                self.ai_processor.stop_processing()
            
            # 关闭线程池
            self.thread_pool.shutdown(wait=True)
            
            logger.info("实时流处理管道已停止")
            
        except Exception as e:
            logger.error(f"停止实时流处理管道失败: {e}")
    
    async def start_camera_stream(self, camera_id: str, ai_algorithms: List[str] = None, 
                                 priority: int = 5) -> bool:
        """启动单个摄像头的实时流处理"""
        try:
            logger.info(f"启动摄像头 {camera_id} 的实时流处理")
            
            # 检查是否已在处理
            if camera_id in self.camera_streams:
                stream_info = self.camera_streams[camera_id]
                if stream_info.status in ["active", "starting"]:
                    logger.warning(f"摄像头 {camera_id} 已在处理中")
                    return True
            
            # 创建流信息
            stream_info = CameraStreamInfo(camera_id, priority)
            stream_info.ai_algorithms = ai_algorithms or self._get_default_ai_algorithms(camera_id)
            stream_info.status = "starting"
            self.camera_streams[camera_id] = stream_info
            
            # 分配工作器
            worker_id = await self._assign_camera_to_worker(camera_id)
            if not worker_id:
                logger.error(f"无法为摄像头 {camera_id} 分配工作器")
                stream_info.status = "error"
                stream_info.last_error = "无法分配工作器"
                return False
            
            stream_info.worker_id = worker_id
            
            # 启动视频流处理
            db = SessionLocal()
            try:
                success = await self.video_service.start_real_time_processing(camera_id, db)
                if success:
                    stream_info.status = "active"
                    stream_info.started_at = datetime.now()
                    stream_info.health_status = "healthy"
                    
                    # 更新性能指标
                    self.performance_metrics["active_cameras"] += 1
                    
                    logger.info(f"摄像头 {camera_id} 实时流处理启动成功")
                    return True
                else:
                    stream_info.status = "error"
                    stream_info.last_error = "视频流启动失败"
                    await self._unassign_camera_from_worker(camera_id)
                    return False
            finally:
                db.close()
                
        except Exception as e:
            logger.error(f"启动摄像头 {camera_id} 实时流处理失败: {e}")
            if camera_id in self.camera_streams:
                self.camera_streams[camera_id].status = "error"
                self.camera_streams[camera_id].last_error = str(e)
            return False
    
    async def stop_camera_stream(self, camera_id: str) -> bool:
        """停止单个摄像头的实时流处理"""
        try:
            logger.info(f"停止摄像头 {camera_id} 的实时流处理")
            
            if camera_id not in self.camera_streams:
                logger.warning(f"摄像头 {camera_id} 未在处理中")
                return True
            
            stream_info = self.camera_streams[camera_id]
            stream_info.status = "stopping"
            
            # 停止视频流处理
            success = await self.video_service.stop_real_time_processing(camera_id)
            
            # 从工作器中移除
            await self._unassign_camera_from_worker(camera_id)
            
            # 移除流信息
            del self.camera_streams[camera_id]
            
            # 更新性能指标
            if self.performance_metrics["active_cameras"] > 0:
                self.performance_metrics["active_cameras"] -= 1
            
            logger.info(f"摄像头 {camera_id} 实时流处理已停止")
            return success
            
        except Exception as e:
            logger.error(f"停止摄像头 {camera_id} 实时流处理失败: {e}")
            return False
    
    async def batch_start_cameras(self, camera_configs: List[Dict[str, Any]]) -> Dict[str, bool]:
        """批量启动多个摄像头的实时流处理"""
        results = {}
        
        # 限制并发启动数量
        semaphore = asyncio.Semaphore(20)  # 最多同时启动20个流
        
        async def start_single_camera(config: Dict[str, Any]):
            async with semaphore:
                camera_id = config["camera_id"]
                ai_algorithms = config.get("ai_algorithms", [])
                priority = config.get("priority", 5)
                
                results[camera_id] = await self.start_camera_stream(
                    camera_id, ai_algorithms, priority
                )
        
        # 并发启动所有摄像头
        tasks = [start_single_camera(config) for config in camera_configs]
        await asyncio.gather(*tasks, return_exceptions=True)
        
        successful = sum(results.values())
        logger.info(f"批量启动摄像头完成，成功: {successful}/{len(camera_configs)}")
        
        return results
    
    async def batch_stop_cameras(self, camera_ids: List[str]) -> Dict[str, bool]:
        """批量停止多个摄像头的实时流处理"""
        results = {}
        
        # 并发停止所有摄像头
        tasks = []
        for camera_id in camera_ids:
            if camera_id in self.camera_streams:
                tasks.append(self.stop_camera_stream(camera_id))
                results[camera_id] = True
            else:
                results[camera_id] = False
        
        if tasks:
            await asyncio.gather(*tasks, return_exceptions=True)
        
        logger.info(f"批量停止摄像头完成，处理: {len(tasks)} 个流")
        return results
    
    async def restart_camera_stream(self, camera_id: str) -> bool:
        """重启摄像头流处理"""
        try:
            logger.info(f"重启摄像头 {camera_id} 的流处理")
            
            # 保存原有配置
            original_config = None
            if camera_id in self.camera_streams:
                stream_info = self.camera_streams[camera_id]
                original_config = {
                    "ai_algorithms": stream_info.ai_algorithms.copy(),
                    "priority": stream_info.priority
                }
            
            # 停止流
            await self.stop_camera_stream(camera_id)
            
            # 等待一小段时间
            await asyncio.sleep(1)
            
            # 重新启动
            if original_config:
                return await self.start_camera_stream(
                    camera_id, 
                    original_config["ai_algorithms"], 
                    original_config["priority"]
                )
            else:
                return await self.start_camera_stream(camera_id)
                
        except Exception as e:
            logger.error(f"重启摄像头 {camera_id} 流处理失败: {e}")
            return False
    
    async def get_pipeline_status(self) -> Dict[str, Any]:
        """获取管道状态"""
        try:
            # 计算运行时长
            uptime = 0.0
            if self.started_at:
                uptime = (datetime.now() - self.started_at).total_seconds()
            
            # 统计摄像头状态
            camera_status_counts = defaultdict(int)
            for stream_info in self.camera_streams.values():
                camera_status_counts[stream_info.status] += 1
            
            # 统计工作器状态
            worker_status_counts = defaultdict(int)
            total_worker_load = 0.0
            for worker in self.stream_workers.values():
                worker_status_counts[worker.status] += 1
                total_worker_load += worker.load_factor
            
            avg_worker_load = total_worker_load / len(self.stream_workers) if self.stream_workers else 0.0
            
            # 获取GPU使用率
            gpu_utilization = 0.0
            try:
                gpu_info = await gpu_resource_manager.get_gpu_info()
                if gpu_info:
                    utilizations = [gpu.utilization for gpu in gpu_info.values()]
                    gpu_utilization = sum(utilizations) / len(utilizations) if utilizations else 0.0
            except:
                pass
            
            return {
                "is_running": self.is_running,
                "uptime_seconds": uptime,
                "uptime_formatted": self._format_uptime(uptime),
                "total_cameras": len(self.camera_streams),
                "camera_status": dict(camera_status_counts),
                "total_workers": len(self.stream_workers),
                "worker_status": dict(worker_status_counts),
                "average_worker_load": avg_worker_load,
                "gpu_utilization": gpu_utilization,
                "performance_metrics": self.performance_metrics.copy(),
                "started_at": self.started_at.isoformat() if self.started_at else None
            }
            
        except Exception as e:
            logger.error(f"获取管道状态失败: {e}")
            return {"error": str(e)}
    
    async def get_camera_status(self, camera_id: str) -> Dict[str, Any]:
        """获取摄像头状态"""
        try:
            if camera_id not in self.camera_streams:
                return {
                    "camera_id": camera_id,
                    "status": "not_found",
                    "message": "摄像头未在处理中"
                }
            
            stream_info = self.camera_streams[camera_id]
            
            # 获取视频流状态
            video_status = await self.video_service.get_real_time_stream_status(camera_id)
            
            # 计算运行时长
            uptime = 0.0
            if stream_info.started_at:
                uptime = (datetime.now() - stream_info.started_at).total_seconds()
            
            return {
                "camera_id": camera_id,
                "status": stream_info.status,
                "priority": stream_info.priority,
                "worker_id": stream_info.worker_id,
                "ai_algorithms": stream_info.ai_algorithms,
                "processing_fps": stream_info.processing_fps,
                "frames_processed": stream_info.frames_processed,
                "error_count": stream_info.error_count,
                "last_error": stream_info.last_error,
                "health_status": stream_info.health_status,
                "uptime_seconds": uptime,
                "uptime_formatted": self._format_uptime(uptime),
                "started_at": stream_info.started_at.isoformat() if stream_info.started_at else None,
                "last_frame_time": stream_info.last_frame_time.isoformat() if stream_info.last_frame_time else None,
                "video_stream_status": video_status
            }
            
        except Exception as e:
            logger.error(f"获取摄像头 {camera_id} 状态失败: {e}")
            return {"camera_id": camera_id, "status": "error", "error": str(e)}
    
    async def update_camera_config(self, camera_id: str, config: Dict[str, Any]) -> bool:
        """更新摄像头配置"""
        try:
            if camera_id not in self.camera_streams:
                logger.warning(f"摄像头 {camera_id} 未在处理中")
                return False
            
            stream_info = self.camera_streams[camera_id]
            
            # 更新AI算法配置
            if "ai_algorithms" in config:
                stream_info.ai_algorithms = config["ai_algorithms"]
            
            # 更新优先级
            if "priority" in config:
                old_priority = stream_info.priority
                stream_info.priority = config["priority"]
                
                # 如果优先级变化较大，可能需要重新分配工作器
                if abs(old_priority - stream_info.priority) > 2:
                    await self._rebalance_camera_assignment(camera_id)
            
            # 更新处理帧率
            if "processing_fps" in config:
                stream_info.processing_fps = config["processing_fps"]
            
            logger.info(f"摄像头 {camera_id} 配置更新成功")
            return True
            
        except Exception as e:
            logger.error(f"更新摄像头 {camera_id} 配置失败: {e}")
            return False    
    
    # Private methods for internal management
    
    async def _create_initial_workers(self):
        """创建初始工作器"""
        try:
            # 根据配置创建工作器
            worker_count = max(1, self.max_workers)
            cameras_per_worker = settings.MAX_CONCURRENT_STREAMS // worker_count
            
            for i in range(worker_count):
                worker_id = f"worker_{i+1}"
                worker = StreamWorker(worker_id, cameras_per_worker)
                self.stream_workers[worker_id] = worker
                
            logger.info(f"创建了 {worker_count} 个工作器，每个处理最多 {cameras_per_worker} 路摄像头")
            
        except Exception as e:
            logger.error(f"创建初始工作器失败: {e}")
            raise
    
    async def _start_worker(self, worker_id: str):
        """启动工作器任务"""
        try:
            if worker_id in self.worker_tasks:
                logger.warning(f"工作器 {worker_id} 任务已存在")
                return
            
            worker = self.stream_workers[worker_id]
            worker.status = "running"
            
            # 创建工作器任务
            task = asyncio.create_task(self._worker_loop(worker_id))
            self.worker_tasks[worker_id] = task
            
            logger.info(f"工作器 {worker_id} 已启动")
            
        except Exception as e:
            logger.error(f"启动工作器 {worker_id} 失败: {e}")
    
    async def _stop_worker(self, worker_id: str):
        """停止工作器任务"""
        try:
            if worker_id not in self.worker_tasks:
                return
            
            task = self.worker_tasks[worker_id]
            task.cancel()
            
            try:
                await task
            except asyncio.CancelledError:
                pass
            
            del self.worker_tasks[worker_id]
            
            if worker_id in self.stream_workers:
                self.stream_workers[worker_id].status = "idle"
            
            logger.info(f"工作器 {worker_id} 已停止")
            
        except Exception as e:
            logger.error(f"停止工作器 {worker_id} 失败: {e}")
    
    async def _worker_loop(self, worker_id: str):
        """工作器主循环"""
        try:
            worker = self.stream_workers[worker_id]
            logger.info(f"工作器 {worker_id} 开始运行")
            
            while self.is_running:
                try:
                    # 更新工作器状态
                    worker.last_activity = datetime.now()
                    
                    # 计算负载因子
                    worker.load_factor = len(worker.assigned_cameras) / worker.max_cameras
                    
                    # 检查是否过载
                    if worker.load_factor > 0.9:
                        worker.status = "overloaded"
                    elif worker.load_factor > 0.7:
                        worker.status = "running"
                    else:
                        worker.status = "idle"
                    
                    # 处理分配给此工作器的摄像头
                    await self._process_worker_cameras(worker_id)
                    
                    # 短暂休眠
                    await asyncio.sleep(1)
                    
                except Exception as e:
                    logger.error(f"工作器 {worker_id} 处理异常: {e}")
                    worker.error_count += 1
                    worker.status = "error"
                    await asyncio.sleep(5)  # 出错时等待更长时间
                    
        except asyncio.CancelledError:
            logger.info(f"工作器 {worker_id} 被取消")
        except Exception as e:
            logger.error(f"工作器 {worker_id} 异常退出: {e}")
        finally:
            logger.info(f"工作器 {worker_id} 已退出")
    
    async def _process_worker_cameras(self, worker_id: str):
        """处理工作器分配的摄像头"""
        try:
            worker = self.stream_workers[worker_id]
            
            # 检查分配给此工作器的摄像头
            for camera_id in list(worker.assigned_cameras):
                if camera_id not in self.camera_streams:
                    # 摄像头已被移除，从工作器中清理
                    worker.assigned_cameras.discard(camera_id)
                    continue
                
                stream_info = self.camera_streams[camera_id]
                
                # 更新摄像头状态
                await self._update_camera_stream_status(camera_id)
                
                # 检查摄像头健康状态
                if stream_info.status == "active":
                    await self._check_camera_health(camera_id)
                
        except Exception as e:
            logger.error(f"处理工作器 {worker_id} 摄像头时发生错误: {e}")
    
    async def _update_camera_stream_status(self, camera_id: str):
        """更新摄像头流状态"""
        try:
            if camera_id not in self.camera_streams:
                return
            
            stream_info = self.camera_streams[camera_id]
            
            # 获取视频流状态
            video_status = await self.video_service.get_real_time_stream_status(camera_id)
            
            if video_status.get("stream_health", False):
                stream_info.health_status = "healthy"
                stream_info.last_frame_time = datetime.now()
                
                # 更新帧计数
                extractor_stats = video_status.get("extractor_stats", {})
                frames_extracted = extractor_stats.get("frames_extracted", 0)
                if frames_extracted > stream_info.frames_processed:
                    stream_info.frames_processed = frames_extracted
            else:
                stream_info.health_status = "unhealthy"
                stream_info.error_count += 1
                
                # 记录错误信息
                if "error" in video_status:
                    stream_info.last_error = video_status["error"]
                
        except Exception as e:
            logger.error(f"更新摄像头 {camera_id} 流状态失败: {e}")
    
    async def _check_camera_health(self, camera_id: str):
        """检查摄像头健康状态"""
        try:
            if camera_id not in self.camera_streams:
                return
            
            stream_info = self.camera_streams[camera_id]
            
            # 检查最后帧时间
            if stream_info.last_frame_time:
                time_since_last_frame = (datetime.now() - stream_info.last_frame_time).total_seconds()
                
                # 如果超过阈值时间没有收到帧，标记为不健康
                if time_since_last_frame > settings.STREAM_TIMEOUT:
                    stream_info.health_status = "unhealthy"
                    stream_info.error_count += 1
                    stream_info.last_error = f"超过 {settings.STREAM_TIMEOUT} 秒未收到帧"
                    
                    # 尝试重启流
                    logger.warning(f"摄像头 {camera_id} 健康检查失败，尝试重启")
                    await self.restart_camera_stream(camera_id)
            
        except Exception as e:
            logger.error(f"检查摄像头 {camera_id} 健康状态失败: {e}")
    
    async def _assign_camera_to_worker(self, camera_id: str) -> Optional[str]:
        """将摄像头分配给工作器"""
        try:
            # 找到负载最低的工作器
            best_worker_id = None
            min_load = float('inf')
            
            for worker_id, worker in self.stream_workers.items():
                if len(worker.assigned_cameras) < worker.max_cameras:
                    if worker.load_factor < min_load:
                        min_load = worker.load_factor
                        best_worker_id = worker_id
            
            if best_worker_id:
                worker = self.stream_workers[best_worker_id]
                worker.assigned_cameras.add(camera_id)
                logger.info(f"摄像头 {camera_id} 分配给工作器 {best_worker_id}")
                return best_worker_id
            else:
                logger.error(f"无可用工作器分配给摄像头 {camera_id}")
                return None
                
        except Exception as e:
            logger.error(f"分配摄像头 {camera_id} 到工作器失败: {e}")
            return None
    
    async def _unassign_camera_from_worker(self, camera_id: str):
        """从工作器中移除摄像头"""
        try:
            if camera_id not in self.camera_streams:
                return
            
            stream_info = self.camera_streams[camera_id]
            worker_id = stream_info.worker_id
            
            if worker_id and worker_id in self.stream_workers:
                worker = self.stream_workers[worker_id]
                worker.assigned_cameras.discard(camera_id)
                logger.info(f"摄像头 {camera_id} 从工作器 {worker_id} 中移除")
            
        except Exception as e:
            logger.error(f"从工作器移除摄像头 {camera_id} 失败: {e}")
    
    async def _rebalance_camera_assignment(self, camera_id: str):
        """重新平衡摄像头分配"""
        try:
            # 从当前工作器移除
            await self._unassign_camera_from_worker(camera_id)
            
            # 重新分配
            new_worker_id = await self._assign_camera_to_worker(camera_id)
            
            if new_worker_id and camera_id in self.camera_streams:
                self.camera_streams[camera_id].worker_id = new_worker_id
                logger.info(f"摄像头 {camera_id} 重新分配给工作器 {new_worker_id}")
            
        except Exception as e:
            logger.error(f"重新平衡摄像头 {camera_id} 分配失败: {e}")
    
    async def _monitoring_loop(self):
        """监控循环"""
        try:
            logger.info("启动管道监控循环")
            
            while self.is_running:
                try:
                    # 更新性能指标
                    await self._update_performance_metrics()
                    
                    # 检查系统负载
                    await self._check_system_load()
                    
                    # 清理不活跃的流
                    await self._cleanup_inactive_streams()
                    
                    # 等待下次监控
                    await asyncio.sleep(30)  # 每30秒监控一次
                    
                except Exception as e:
                    logger.error(f"监控循环异常: {e}")
                    await asyncio.sleep(10)
                    
        except asyncio.CancelledError:
            logger.info("监控循环被取消")
        except Exception as e:
            logger.error(f"监控循环异常退出: {e}")
    
    async def _load_balancing_loop(self):
        """负载均衡循环"""
        try:
            logger.info("启动负载均衡循环")
            
            while self.is_running:
                try:
                    # 检查工作器负载
                    await self._check_worker_load_balance()
                    
                    # 动态调整工作器数量
                    await self._adjust_worker_count()
                    
                    # 等待下次负载均衡
                    await asyncio.sleep(60)  # 每分钟检查一次
                    
                except Exception as e:
                    logger.error(f"负载均衡循环异常: {e}")
                    await asyncio.sleep(30)
                    
        except asyncio.CancelledError:
            logger.info("负载均衡循环被取消")
        except Exception as e:
            logger.error(f"负载均衡循环异常退出: {e}")
    
    async def _health_check_loop(self):
        """健康检查循环"""
        try:
            logger.info("启动健康检查循环")
            
            while self.is_running:
                try:
                    # 检查所有摄像头健康状态
                    for camera_id in list(self.camera_streams.keys()):
                        await self._check_camera_health(camera_id)
                    
                    # 检查工作器健康状态
                    await self._check_worker_health()
                    
                    # 等待下次健康检查
                    await asyncio.sleep(45)  # 每45秒检查一次
                    
                except Exception as e:
                    logger.error(f"健康检查循环异常: {e}")
                    await asyncio.sleep(15)
                    
        except asyncio.CancelledError:
            logger.info("健康检查循环被取消")
        except Exception as e:
            logger.error(f"健康检查循环异常退出: {e}")
    
    async def _update_performance_metrics(self):
        """更新性能指标"""
        try:
            # 统计摄像头数量
            self.performance_metrics["total_cameras"] = len(self.camera_streams)
            active_count = sum(1 for s in self.camera_streams.values() if s.status == "active")
            self.performance_metrics["active_cameras"] = active_count
            
            # 统计总帧数
            total_frames = sum(s.frames_processed for s in self.camera_streams.values())
            self.performance_metrics["total_frames_processed"] = total_frames
            
            # 计算平均FPS
            if active_count > 0:
                total_fps = sum(s.processing_fps for s in self.camera_streams.values() if s.status == "active")
                self.performance_metrics["average_fps"] = total_fps / active_count
            else:
                self.performance_metrics["average_fps"] = 0.0
            
            # 计算系统负载
            total_workers = len(self.stream_workers)
            if total_workers > 0:
                total_load = sum(w.load_factor for w in self.stream_workers.values())
                self.performance_metrics["system_load"] = total_load / total_workers
            else:
                self.performance_metrics["system_load"] = 0.0
            
            # 获取GPU利用率
            try:
                gpu_info = await gpu_resource_manager.get_gpu_info()
                if gpu_info:
                    utilizations = [gpu.utilization for gpu in gpu_info.values()]
                    self.performance_metrics["gpu_utilization"] = sum(utilizations) / len(utilizations)
                else:
                    self.performance_metrics["gpu_utilization"] = 0.0
            except:
                self.performance_metrics["gpu_utilization"] = 0.0
            
            # 计算错误率
            total_errors = sum(s.error_count for s in self.camera_streams.values())
            if total_frames > 0:
                self.performance_metrics["error_rate"] = total_errors / total_frames
            else:
                self.performance_metrics["error_rate"] = 0.0
            
            # 计算运行时长
            if self.started_at:
                self.performance_metrics["uptime"] = (datetime.now() - self.started_at).total_seconds()
            
        except Exception as e:
            logger.error(f"更新性能指标失败: {e}")
    
    async def _check_system_load(self):
        """检查系统负载"""
        try:
            system_load = self.performance_metrics["system_load"]
            
            # 如果系统负载过高，降低处理帧率
            if system_load > 0.9:
                logger.warning(f"系统负载过高: {system_load:.2f}，降低处理帧率")
                await self._reduce_processing_fps()
            elif system_load < 0.5:
                # 如果系统负载较低，可以提高处理帧率
                await self._increase_processing_fps()
            
        except Exception as e:
            logger.error(f"检查系统负载失败: {e}")
    
    async def _reduce_processing_fps(self):
        """降低处理帧率"""
        try:
            for stream_info in self.camera_streams.values():
                if stream_info.processing_fps > 1:
                    stream_info.processing_fps = max(1, stream_info.processing_fps - 1)
            
            logger.info("已降低所有摄像头的处理帧率")
            
        except Exception as e:
            logger.error(f"降低处理帧率失败: {e}")
    
    async def _increase_processing_fps(self):
        """提高处理帧率"""
        try:
            max_fps = settings.VIDEO_PROCESSING_FPS
            
            for stream_info in self.camera_streams.values():
                if stream_info.processing_fps < max_fps:
                    stream_info.processing_fps = min(max_fps, stream_info.processing_fps + 1)
            
            logger.info("已提高所有摄像头的处理帧率")
            
        except Exception as e:
            logger.error(f"提高处理帧率失败: {e}")
    
    async def _cleanup_inactive_streams(self):
        """清理不活跃的流"""
        try:
            inactive_cameras = []
            
            for camera_id, stream_info in self.camera_streams.items():
                # 检查长时间处于错误状态的摄像头
                if stream_info.status == "error" and stream_info.error_count > 10:
                    inactive_cameras.append(camera_id)
                
                # 检查长时间没有活动的摄像头
                if stream_info.last_frame_time:
                    inactive_time = (datetime.now() - stream_info.last_frame_time).total_seconds()
                    if inactive_time > 300:  # 5分钟没有活动
                        inactive_cameras.append(camera_id)
            
            # 清理不活跃的摄像头
            for camera_id in inactive_cameras:
                logger.warning(f"清理不活跃的摄像头: {camera_id}")
                await self.stop_camera_stream(camera_id)
            
        except Exception as e:
            logger.error(f"清理不活跃流失败: {e}")
    
    async def _check_worker_load_balance(self):
        """检查工作器负载均衡"""
        try:
            if not self.stream_workers:
                return
            
            # 计算负载分布
            loads = [w.load_factor for w in self.stream_workers.values()]
            max_load = max(loads)
            min_load = min(loads)
            
            # 如果负载差异过大，进行重新平衡
            if max_load - min_load > 0.3:
                logger.info(f"工作器负载不均衡，最大: {max_load:.2f}，最小: {min_load:.2f}")
                await self._rebalance_workers()
            
        except Exception as e:
            logger.error(f"检查工作器负载均衡失败: {e}")
    
    async def _rebalance_workers(self):
        """重新平衡工作器"""
        try:
            # 找到负载最高和最低的工作器
            max_worker = max(self.stream_workers.values(), key=lambda w: w.load_factor)
            min_worker = min(self.stream_workers.values(), key=lambda w: w.load_factor)
            
            # 如果差异足够大，移动一些摄像头
            if max_worker.load_factor - min_worker.load_factor > 0.3:
                # 从高负载工作器移动摄像头到低负载工作器
                cameras_to_move = list(max_worker.assigned_cameras)[:2]  # 移动2个摄像头
                
                for camera_id in cameras_to_move:
                    if len(min_worker.assigned_cameras) < min_worker.max_cameras:
                        max_worker.assigned_cameras.discard(camera_id)
                        min_worker.assigned_cameras.add(camera_id)
                        
                        if camera_id in self.camera_streams:
                            self.camera_streams[camera_id].worker_id = min_worker.worker_id
                        
                        logger.info(f"摄像头 {camera_id} 从工作器 {max_worker.worker_id} 移动到 {min_worker.worker_id}")
            
        except Exception as e:
            logger.error(f"重新平衡工作器失败: {e}")
    
    async def _adjust_worker_count(self):
        """动态调整工作器数量"""
        try:
            current_workers = len(self.stream_workers)
            active_cameras = self.performance_metrics["active_cameras"]
            
            # 计算理想工作器数量
            ideal_workers = max(1, (active_cameras + 99) // 100)  # 每100个摄像头一个工作器
            
            if ideal_workers > current_workers and current_workers < self.max_workers:
                # 需要增加工作器
                new_worker_id = f"worker_{current_workers + 1}"
                worker = StreamWorker(new_worker_id, 100)
                self.stream_workers[new_worker_id] = worker
                await self._start_worker(new_worker_id)
                logger.info(f"增加工作器: {new_worker_id}")
                
            elif ideal_workers < current_workers and current_workers > 1:
                # 需要减少工作器（只在负载很低时）
                avg_load = sum(w.load_factor for w in self.stream_workers.values()) / current_workers
                if avg_load < 0.3:
                    # 找到负载最低的工作器
                    min_worker = min(self.stream_workers.values(), key=lambda w: w.load_factor)
                    
                    # 将其摄像头重新分配
                    for camera_id in list(min_worker.assigned_cameras):
                        await self._rebalance_camera_assignment(camera_id)
                    
                    # 停止并移除工作器
                    await self._stop_worker(min_worker.worker_id)
                    del self.stream_workers[min_worker.worker_id]
                    logger.info(f"移除工作器: {min_worker.worker_id}")
            
        except Exception as e:
            logger.error(f"调整工作器数量失败: {e}")
    
    async def _check_worker_health(self):
        """检查工作器健康状态"""
        try:
            current_time = datetime.now()
            
            for worker_id, worker in self.stream_workers.items():
                # 检查工作器是否长时间没有活动
                inactive_time = (current_time - worker.last_activity).total_seconds()
                
                if inactive_time > 120:  # 2分钟没有活动
                    logger.warning(f"工作器 {worker_id} 长时间无活动，重启任务")
                    
                    # 重启工作器任务
                    await self._stop_worker(worker_id)
                    await self._start_worker(worker_id)
                
                # 检查错误率
                if worker.error_count > 50:
                    logger.warning(f"工作器 {worker_id} 错误率过高，重置错误计数")
                    worker.error_count = 0
            
        except Exception as e:
            logger.error(f"检查工作器健康状态失败: {e}")
    
    def _get_default_ai_algorithms(self, camera_id: str) -> List[str]:
        """获取摄像头的默认AI算法配置"""
        try:
            # 这里可以根据摄像头位置、类型等配置不同的算法
            # 暂时返回基础算法组合
            return [
                "person_detection",
                "behavior_analysis", 
                "crowd_density"
            ]
        except Exception as e:
            logger.error(f"获取摄像头 {camera_id} 默认AI算法失败: {e}")
            return ["person_detection"]
    
    def _format_uptime(self, seconds: float) -> str:
        """格式化运行时长"""
        hours = int(seconds // 3600)
        minutes = int((seconds % 3600) // 60)
        secs = int(seconds % 60)
        return f"{hours:02d}:{minutes:02d}:{secs:02d}"


# 全局实例
_pipeline_instance: Optional[RealTimeStreamPipeline] = None


def get_pipeline() -> RealTimeStreamPipeline:
    """获取实时流处理管道实例"""
    global _pipeline_instance
    if _pipeline_instance is None:
        _pipeline_instance = RealTimeStreamPipeline()
    return _pipeline_instance


async def start_pipeline():
    """启动实时流处理管道"""
    pipeline = get_pipeline()
    await pipeline.initialize()
    await pipeline.start()


async def stop_pipeline():
    """停止实时流处理管道"""
    pipeline = get_pipeline()
    await pipeline.stop()